﻿/// <summary>
/// FileName: Tokenizer.cs
/// Author: Jiang Xiaolong
/// Created Time: 2015.05.12
/// Version: 1.0
/// Company: Sunnytech
/// Function: 分词器，实现分词细节；
///
/// Changed By:
/// Modification Time:
/// Discription:
/// </summary>
using System.Collections.Generic;
using Expression.api;

namespace Expression.impl
{
	public class Tokenizer : ForwardQuery<Token>
	{
		//公式字符串读取器；
		private TokenReader _inputExp;

		//小数点
		private char _decimalSeparator = '.';
		//行注释
		private string _lineComment = "//";
		//块注释起始
		private string _blockCommentStart = "/*";
		//块注释末尾
		private string _blockCommentEnd = "*/";
		//括号
		private char[] _brackets = new char[] { '(', '[', '{', '}', ']', ')', '（', '）', '【', '】' };
		//是否将"|"当做括号处理
		private bool _treatSinglePipeAsBracket = true;
		//一些特殊的ID起始标志位，像“$Test”中的“$”
		private List<char> _specialIdStarters = new List<char>();
		//一些特殊ID结尾，像“Test：”中的“:”
		private List<char> _specialIdTerminators = new List<char>();
		//系统关键词记录，如果关键词大小写不敏感，则都用小写记录；
		private Dictionary<string, string> _keywords = new Dictionary<string, string>();
		//关键词是否大小写敏感
		private bool _keywordsCaseSensitive = false;
		//字符分隔符，用于字符串常量的分隔，这里主要是“\'”和“\"”
		private Dictionary<char, char> _stringDelimiters = new Dictionary<char, char>();
		//bool类型常量
		private List<string> _boolConstant = new List<string>();

		public Tokenizer(string input_exp)
		{
			this._inputExp = new TokenReader(input_exp);
			_stringDelimiters.Clear();
			_stringDelimiters.Add('\"', '\\');
			_stringDelimiters.Add('\'', '\0');
			_boolConstant.Clear();
			_boolConstant.Add("false");
			_boolConstant.Add("true");
		}

		/// <summary>
		/// 是否为输入公式的末尾；
		/// </summary>
		/// <returns>是否为输入公式的末尾；</returns>
		protected override Token EndOfInput()
		{
			return Token.CreateAndFill(Token.TokenType.EOI, _inputExp.current());
		}

		/// <summary>
		/// 获取分词；
		/// </summary>
		/// <returns>当前分词；</returns>
		protected override Token Fetch()
		{
			// 获取并且消除空格
			while (_inputExp.current().IsWhiteSpace())
			{
				_inputExp.consume();
			}

			// 为行尾则返回空
			if (_inputExp.current().IsEndOfInput())
			{
				return null;
			}

			// 行注释处理
			if (IsAtStartOfLineComment(true))
			{
				SkipToEndOfLine();
				return Fetch();
			}

			// 注释块处理
			if (IsAtStartOfBlockComment(true))
			{
				SkipBlockComment();
				return Fetch();
			}

			// 是否以数字起始，获取常量
			if (IsAtStartOfNumber())
			{
				return FetchNumber();
			}

			// 是否为字母开头，获取变量和函数
			if (IsAtStartOfIdentifier())
			{
				return FetchId();
			}

			// 字符串常量识别
			if (_stringDelimiters.ContainsKey(_inputExp.current().GetValue()))
			{
				return FetchString();
			}

			//括号处理
			if (IsAtBracket(false))
			{
				return Token.CreateAndFill(Token.TokenType.OPERATOR, _inputExp.consume());
			}

			// 检测是否在特殊字符的开头，一些特殊的ID起始标志位，像“$Test”中的“$”
			if (IsAtStartOfSpecialId())
			{
				return FetchSpecialId();
			}

			// 运算符处理
			if (IsOperator(_inputExp.current()))
			{
				return FetchOperator();
			}

			//当前输入有问题，一般不应该运行到这一步
			problemCollector.Add(ParseError.Error(_inputExp.current(), "无效的输入字符: " + _inputExp.current().GetStringValue()));

			_inputExp.consume();
			return Fetch();
		}

		/// <summary>
		/// 判断是否定义了行注释标志符，若定义了，进入子流程，判断字符串是否为行注释标识符；
		/// </summary>
		/// <param name="consume">是否直接销毁行注释符；/param>
		/// <returns>是否有行注释符；</returns>
		private bool IsAtStartOfLineComment(bool consume)
		{
			if (_lineComment != "")
			{
				return CanConsumeThisString(_lineComment, consume);
			}
			else
			{
				return false;
			}
		}

		/// <summary>
		/// 检测当前字符是否为指定字符串，且检测后要不要销毁；
		/// </summary>
		/// <param name="check_string">定义好的字符串；</param>
		/// <param name="consume">是否直接销毁；</param>
		/// <returns>是否匹配；</returns>
		private bool CanConsumeThisString(string check_string, bool consume)
		{
			for (int i = 0; i < check_string.Length; i++)
			{
				//利用next向后对比，看是否为行注释符“//”
				if (!(_inputExp.next(i).Is(check_string[i])))
				{
					return false;
				}
			}
			if (consume)
			{
				_inputExp.consume(check_string.Length);
			}
			return true;
		}

		/// <summary>
		/// 如果有行注释，则忽略行注释后的所有字符，直至遇到换行符或行尾；
		/// </summary>
		private void SkipToEndOfLine()
		{
			while (!_inputExp.current().IsEndOfInput() && !_inputExp.current().IsNewLine())
			{
				_inputExp.consume();
			}
		}

		/// <summary>
		/// 判断是否为块注释起始位置；
		/// </summary>
		/// <param name="consume">是否销毁该符号；</param>
		/// <returns>是否匹配块注释；</returns>
		private bool IsAtStartOfBlockComment(bool consume)
		{
			return CanConsumeThisString(_blockCommentStart, consume);
		}

		/// <summary>
		/// 跳过块注释；
		/// </summary>
		private void SkipBlockComment()
		{
			while (!_inputExp.current().IsEndOfInput())
			{
				if (IsAtEndOfBlockComment())
				{
					return;
				}
				_inputExp.consume();
			}
			problemCollector.Add(ParseError.Error(_inputExp.current(), "块注释没有结束符！！！"));
		}

		/// <summary>
		/// 是否为块注释结尾；
		/// </summary>
		/// <returns></returns>
		private bool IsAtEndOfBlockComment()
		{
			return CanConsumeThisString(_blockCommentEnd, true);
		}

		/// <summary>
		/// 判断当前字符是否为数字;
		/// </summary>
		/// <returns>是否为数字；</returns>
		private bool IsAtStartOfNumber()
		{
			if (_inputExp.current().IsDigit())
			{
				return true;
			}
			//else if (_inputExp.current().Is('-') && _inputExp.next().IsDigit())
			//{
			//	return true;
			//}
			//else if (_inputExp.current().Is('-') && _inputExp.next().Is('.') && _inputExp.next(2).IsDigit())
			//{
			//	return true;
			//}
			else if (_inputExp.current().Is('.') && _inputExp.next().IsDigit())
			{
				return true;
			}
			else
			{
				return false;
			}
		}

		/// <summary>
		/// 读取完整的数字；
		/// </summary>
		/// <returns>当前分词；</returns>
		private Token FetchNumber()
		{
			Token result = Token.Create(Token.TokenType.INTEGER, _inputExp.current());
			result.AddToContent(_inputExp.consume());
			while (_inputExp.current().IsDigit() || _inputExp.current().Is(_decimalSeparator))
			{
				if (_inputExp.current().Is(_decimalSeparator))
				{
					if (result.Is(Token.TokenType.FLOAT) || result.Is(Token.TokenType.DOUBLE))
					{
						//已经有小数点了
						problemCollector.Add(ParseError.Error(_inputExp.current(), "公式中有多个小数点！"));
					}
					else
					{
						Token decimalToken = Token.Create(Token.TokenType.FLOAT, result);
						decimalToken.SetContent(result.GetContents());
						decimalToken.SetSource(result.GetSource());
						result = decimalToken;
					}
					result.AddToContent(_inputExp.consume());
				}
				else
				{
					if (result.IsFloat() && result.GetContents().Length >= 7)
					{
						//检测float有效数字是否超过7位，若满足，则转成double；
						string floatNum = result.GetContents();
						if (floatNum.StartsWith("-"))
						{
							if (floatNum[1] == '0')
							{
								if (floatNum.Length >= 10)
								{
									result.SetTokenType(Token.TokenType.DOUBLE);
								}
							}
							else if (floatNum[1] == '.')
							{
								if (floatNum.Length >= 9)
								{
									result.SetTokenType(Token.TokenType.DOUBLE);
								}
							}
							else
							{
								if (floatNum.Length >= 8)
								{
									result.SetTokenType(Token.TokenType.DOUBLE);
								}
							}
						}
						else
						{
							if (floatNum.StartsWith("0"))
							{
								if (floatNum.Length >= 9)
								{
									result.SetTokenType(Token.TokenType.DOUBLE);
								}
							}
							else
							{
								if (floatNum.Length >= 8)
								{
									result.SetTokenType(Token.TokenType.DOUBLE);
								}
							}
						}
					}
					result.AddToContent(_inputExp.consume());
				}
			}

			return result;
		}

		/// <summary>
		/// 是否为字母或者下划线开始；
		/// </summary>
		/// <returns>是否满足；</returns>
		private bool IsAtStartOfIdentifier()
		{
			return _inputExp.current().IsLetter() || _inputExp.current().Is('_');
		}

		/// <summary>
		/// 获取变量ID或者自定义函数名；
		/// </summary>
		/// <returns>当前分词；</returns>
		private Token FetchId()
		{
			Token result = Token.Create(Token.TokenType.ID, _inputExp.current());
			result.AddToContent(_inputExp.consume());
			while (IsIdentifierChar(_inputExp.current()))
			{
				result.AddToContent(_inputExp.consume());
			}
			if (!_inputExp.current().IsEndOfInput() && _specialIdTerminators.Contains(_inputExp.current().GetValue()))
			{
				Token specialId = Token.Create(Token.TokenType.SPECIAL_ID, result);
				specialId.SetTrigger(_inputExp.current().GetStringValue());
				specialId.SetContent(result.GetContents());
				specialId.SetSource(result.GetContents());
				specialId.AddToSource(_inputExp.current());
				_inputExp.consume();
				return HandleKeywords(specialId);
			}
			return HandleKeywords(result);
		}

		/// <summary>
		/// 检测当前分词是否为关键词；
		/// </summary>
		/// <param name="id_token">当前分词；</param>
		/// <returns>关键词处理以后的当前分词；</returns>
		private Token HandleKeywords(Token id_token)
		{
			string keyword = "";
			if (_keywordsCaseSensitive)
			{
				if (_keywords.ContainsKey(id_token.GetContents()))
				{
					keyword = _keywords[id_token.GetContents()];
				}
			}
			else
			{
				if (_keywords.ContainsKey(id_token.GetContents().ToLower()))
				{
					keyword = _keywords[id_token.GetContents().ToLower()];
				}
			}
			if (keyword != "")  //关键字处理
			{
				Token keywordToken = Token.Create(Token.TokenType.KEYWORD, id_token);
				keywordToken.SetTrigger(keyword);
				keywordToken.SetContent(id_token.GetContents());
				keywordToken.SetSource(id_token.GetSource());
				return keywordToken;
			}
			else if (!id_token.Is(Token.TokenType.SPECIAL_ID)) //bool类型处理
			{
				keyword = id_token.GetContents().ToLower();
				if (_boolConstant.Contains(keyword))
				{
					id_token.SetTokenType(Token.TokenType.BOOL);
				}
			}

			return id_token;
		}

		/// <summary>
		/// 变量获取，数字、字母、下划线；
		/// </summary>
		/// <param name="current">当前读取字符；</param>
		/// <returns>是否为数字、字母、下划线；</returns>
		private bool IsIdentifierChar(ExpChar current)
		{
			return current.IsDigit() || current.IsLetter() || current.Is('_');
		}

		/// <summary>
		/// 获取字符串常量；
		/// </summary>
		/// <returns>当前分词；</returns>
		private Token FetchString()
		{
			//读进来的分词标志
			char separator = _inputExp.current().GetValue();
			//字典中对应的分词标志
			//char escapeChar = _stringDelimiters[_inputExp.current().GetValue()];
			Token result = Token.Create(Token.TokenType.STRING, _inputExp.current());
			result.AddToTrigger(_inputExp.consume());
			while (!_inputExp.current().IsNewLine() && !_inputExp.current().Is(separator) && !_inputExp.current().IsEndOfInput())
			{
				result.AddToContent(_inputExp.consume());
			}
			if (_inputExp.current().Is(separator))
			{
				result.AddToSource(_inputExp.consume());
			}
			else
			{
				problemCollector.Add(ParseError.Error(_inputExp.current(), "常量字符串没有结束分隔符！"));
			}
			return result;
		}

		/// <summary>
		/// 检测是否在括号开头处；
		/// </summary>
		/// <param name="in_symbol"></param>
		/// <returns></returns>
		private bool IsAtBracket(bool in_symbol)
		{
			if (_inputExp.current().Is(_brackets))
			{
				return true;
			}
			if (!in_symbol && _treatSinglePipeAsBracket && _inputExp.current().Is('|') && !_inputExp.next().Is('|'))
			{
				return true;
			}
			return false;
		}


		/// <summary>
		/// 是否在特殊符号的开头处，一些特殊的ID起始标志位，像“$Test”中的“$”；
		/// </summary>
		/// <returns>是否在特殊符号的开头处；</returns>
		private bool IsAtStartOfSpecialId()
		{
			return _specialIdStarters.Contains(_inputExp.current().GetValue());
		}

		/// <summary>
		/// 获取特殊变量ID；
		/// </summary>
		/// <returns>当前分词；</returns>
		private Token FetchSpecialId()
		{
			Token result = Token.Create(Token.TokenType.SPECIAL_ID, _inputExp.current());
			result.AddToTrigger(_inputExp.consume());
			while (IsIdentifierChar(_inputExp.current()))
			{
				result.AddToContent(_inputExp.consume());
			}
			return HandleKeywords(result);
		}

		/// <summary>
		/// 判断是否为运算符；
		/// </summary>
		/// <param name="ch">当前输入字符；</param>
		/// <returns>是否为运算符；</returns>
		private bool IsOperator(ExpChar ch)
		{
			if (ch.IsEndOfInput() || ch.IsDigit() || ch.IsLetter() || ch.IsWhiteSpace())
			{
				return false;
			}

			char c = ch.GetValue();
			if (char.IsControl(c))
			{
				return false;
			}

			if (IsAtBracket(true) || IsAtStartOfBlockComment(false) || IsAtStartOfLineComment(false) || IsAtStartOfNumber() || IsAtStartOfIdentifier() || _stringDelimiters.ContainsKey(ch.GetValue()))
			{
				return false;
			}
			return true;
		}

		/// <summary>
		/// 获取运算符,TODO：是否要处理*=、/=、+=、-=；
		/// </summary>
		/// <returns>当前分词；</returns>
		private Token FetchOperator()
		{
			Token result = Token.Create(Token.TokenType.OPERATOR, _inputExp.current());
			result.AddToTrigger(_inputExp.consume());
			if (result.IsOperator("*") && _inputExp.current().Is('*'))
			{
				result.AddToTrigger(_inputExp.consume());
			}
			else if (result.IsOperator("&") && _inputExp.current().Is('&'))
			{
				result.AddToTrigger(_inputExp.consume());
			}
			else if (result.IsOperator("|") && _inputExp.current().Is('|'))
			{
				result.AddToTrigger(_inputExp.consume());
			}
			else if (result.IsOperator("<") && _inputExp.current().Is('='))
			{
				result.AddToTrigger(_inputExp.consume());
			}
			else if (result.IsOperator(">") && _inputExp.current().Is('='))
			{
				result.AddToTrigger(_inputExp.consume());
			}
			else if (result.IsOperator("!") && _inputExp.current().Is('='))
			{
				result.AddToTrigger(_inputExp.consume());
			}
			else if (result.IsOperator("=") && _inputExp.current().Is('='))
			{
				result.AddToTrigger(_inputExp.consume());
			}
			return result;
		}

		/// <summary>
		/// 错误处理；
		/// </summary>
		/// <param name="pos">错误定位；</param>
		/// <param name="message">错误信息；</param>
		/// <param name="parameters">错误关键信息；</param>
		public void AddError(PositionLocate pos, string message, params object[] parameters)
		{
			GetProblemCollector().Add(ParseError.Error(pos, string.Format(message, parameters)));
		}

		/// <summary>
		/// 警告处理；
		/// </summary>
		/// <param name="pos">警告定位；</param>
		/// <param name="message">警告信息；</param>
		/// <param name="parameters">警告关键信息；</param>
		public void AddWarning(PositionLocate pos, string message, params object[] parameters)
		{
			GetProblemCollector().Add(ParseError.Warning(pos, string.Format(message, parameters)));
		}
	}

}
